#ifndef RS_MWIKI_H
#define RS_MWIKI_H
#define _CRT_SECURE_NO_DEPRECATE 1
#include <hash_map>
#include "rs_types.h"
#include "rs_string.h"
#include "rs_table.h"
#include "rs_token.h"
using namespace std;

struct mwiki_entry {
    // few definitions: 
    //   entry: anything that has <title>
    //   other : entry that has #REDIRECT in text or category or disambiguation in title
    //   article: entry that is not other
    int   mwe_entry_no, mwe_article_no;
    int64 mwe_title_offset;
    int   mwe_title_length;
    int64 mwe_text_offset;
    int   mwe_text_length;
    bool  mwe_is_other;
    wstring mwe_title_contents;
};


class mwiki {
public:
    mwiki (const rs_string& wiki_xml_fname);

    bool create_store_index ();
    bool process_wikipedia (
        const rs_string& art_prefix,
        int start_ix,
        int max_amount,
        int reserve_amount);
    bool read_index  (bool articles_only, int max_size);
    bool write_binary_index ();
    bool read_binary_index (unsigned start, unsigned amount);
    bool write_freq  () {
        return mw_freq.write (mw_freq_fname, L"", L"", L"-fr_all");
    };
    int mw_ctx_size;  // size of one context
    unsigned mw_ctx_count; // number of contexts
    int mw_total; // total number of words (sum of all fd_freq for every word form in mw_freq)
private:
    rs_table mw_freq; // 1 word column, 4 numeric columns for frequencies
    text_cix mw_col_word;
    numb_cix mw_col_lc, mw_col_uc, mw_col_ic, mw_col_mc, mw_col_all;
    //rs_table mw_index; // rs_table is not yet powerful enough for this:
      // mising is support for int64  and for incremental storing of every row as created
      // so the following definition is used
    vector<mwiki_entry> mw_index;
    rs_ostream *mw_cfs; // for context
      // contexts are not held in RAM as they can take much space
    rs_string mw_xml_fname;
    rs_string mw_index_txt_fname;
    rs_string mw_index_bin_fname;
    rs_string mw_freq_fname;
    rs_string mw_ctx_fname;

    void process_wiki_word (
        rs_token& word_token,
        const tokenized_text& tok_text,
        size_t word_ix,
        int article_no);
    void process_wiki_article (
        utf8_string& title_str,
        utf8_string& text_str,
        const rs_string& base_fname,
        int title_no,
        bool store_text,
        bool store_freq);


};



bool clean_all_wiki_markup (int title_no, string& text_str, bool retain_replaced=true);


#endif
